{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "p123.1：有可能导致数值不稳定，导致数值过大，结果变成inf，使softmax无法计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "exp(): argument 'input' (position 1) must be Tensor, not int",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mTypeError\u001b[39m                                 Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtorch\u001b[39;00m\n\u001b[32m      2\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mnp\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[43mtorch\u001b[49m\u001b[43m.\u001b[49m\u001b[43mexp\u001b[49m\u001b[43m(\u001b[49m\u001b[32;43m50\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "\u001b[31mTypeError\u001b[39m: exp(): argument 'input' (position 1) must be Tensor, not int"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "torch.exp(50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "p123.2交叉熵计算为L = - sum( y_i * log(p_i) )，当 p_i接近 0 时，log(p_i )→−∞，出现 log(0) = -inf,导致梯度崩溃"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一道问题的解决：我们可以减去输入向量的最大值来防止溢出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入: tensor([50., 60., 70.])\n",
      "稳定 softmax: tensor([2.0611e-09, 4.5398e-05, 9.9995e-01])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "def stable_softmax(x: torch.Tensor) -> torch.Tensor:\n",
    "    x_max = x.max(dim=-1, keepdim=True).values  # 取最大值\n",
    "    x_shifted = x - x_max                       # 减去最大值，防止溢出\n",
    "    exp_x = torch.exp(x_shifted)                # 再取 exp\n",
    "    return exp_x / exp_x.sum(dim=-1, keepdim=True)\n",
    "\n",
    "x = torch.tensor([50., 60., 70.])\n",
    "print('输入:', x)\n",
    "print('稳定 softmax:', stable_softmax(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第二道问题：可以使用 PyTorch 内置的 torch.nn.CrossEntropyLoss，它内部已做了 log-softmax + nll 的数值稳定组合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "官方 CrossEntropyLoss: 4.541770613286644e-05\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "logits = torch.tensor([[50., 60., 70.],  \n",
    "                       [100., 90., 80.]])\n",
    "targets = torch.tensor([2, 0]) \n",
    "ce = torch.nn.CrossEntropyLoss()\n",
    "loss2 = ce(logits, targets)\n",
    "print('官方 CrossEntropyLoss:', loss2.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "p126.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Val Accuracy: 38.50%\n",
      "Epoch 2, Val Accuracy: 39.10%\n",
      "Epoch 3, Val Accuracy: 39.30%\n",
      "Epoch 4, Val Accuracy: 39.00%\n",
      "Epoch 5, Val Accuracy: 39.40%\n",
      "Test Accuracy: 39.67%\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader, random_split\n",
    "\n",
    "# 定义 Softmax 回归模型\n",
    "class SoftmaxRegression(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(SoftmaxRegression, self).__init__()\n",
    "        self.fc = nn.Linear(input_dim, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)  # 展平输入\n",
    "        x = self.fc(x)\n",
    "        return torch.softmax(x, dim=1)\n",
    "\n",
    "# 定义超参数（仅测试一组超参数）\n",
    "batch_size = 64\n",
    "epochs = 5\n",
    "learning_rate = 0.01\n",
    "\n",
    "# 定义设备\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# 数据预处理\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,), (0.5,))\n",
    "])\n",
    "\n",
    "# 加载数据集（仅使用前 5000 个训练样本）\n",
    "train_dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transform)\n",
    "train_dataset = torch.utils.data.Subset(train_dataset, indices=range(5000))  # 快速测试\n",
    "test_dataset = datasets.MNIST(root='./data', train=False, download=True, transform=transform)\n",
    "\n",
    "# 划分训练集和验证集（80% 训练，20% 验证）\n",
    "train_size = int(0.8 * len(train_dataset))\n",
    "val_size = len(train_dataset) - train_size\n",
    "train_dataset, val_dataset = random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "# 创建 DataLoader\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "# 初始化模型、损失函数和优化器\n",
    "model = SoftmaxRegression(784, 10).to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# 训练和验证\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    for data, target in train_loader:\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    # 验证集评估\n",
    "    model.eval()\n",
    "    val_correct = 0\n",
    "    val_total = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in val_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            _, predicted = torch.max(output, 1)\n",
    "            val_total += target.size(0)\n",
    "            val_correct += (predicted == target).sum().item()\n",
    "    val_accuracy = 100 * val_correct / val_total\n",
    "    print(f'Epoch {epoch + 1}, Val Accuracy: {val_accuracy:.2f}%')\n",
    "\n",
    "# 测试集评估\n",
    "test_correct = 0\n",
    "test_total = 0\n",
    "with torch.no_grad():\n",
    "    for data, target in test_loader:\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        output = model(data)\n",
    "        _, predicted = torch.max(output, 1)\n",
    "        test_total += target.size(0)\n",
    "        test_correct += (predicted == target).sum().item()\n",
    "test_accuracy = 100 * test_correct / test_total\n",
    "print(f'Test Accuracy: {test_accuracy:.2f}%')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Val Accuracy: 30.30%\n",
      "Epoch 2, Val Accuracy: 30.40%\n",
      "Epoch 3, Val Accuracy: 30.40%\n",
      "Epoch 4, Val Accuracy: 28.40%\n",
      "Epoch 5, Val Accuracy: 29.30%\n",
      "Epoch 6, Val Accuracy: 30.70%\n",
      "Epoch 7, Val Accuracy: 30.60%\n",
      "Epoch 8, Val Accuracy: 30.80%\n",
      "Epoch 9, Val Accuracy: 30.60%\n",
      "Epoch 10, Val Accuracy: 29.70%\n",
      "Epoch 11, Val Accuracy: 30.60%\n",
      "Epoch 12, Val Accuracy: 30.90%\n",
      "Epoch 13, Val Accuracy: 30.40%\n",
      "Epoch 14, Val Accuracy: 30.60%\n",
      "Epoch 15, Val Accuracy: 30.60%\n",
      "Epoch 16, Val Accuracy: 30.50%\n",
      "Epoch 17, Val Accuracy: 30.70%\n",
      "Early stopping at epoch 17\n",
      "Test Accuracy: 28.99%\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader, random_split\n",
    "\n",
    "# 定义 Softmax 回归模型\n",
    "class SoftmaxRegression(nn.Module):\n",
    "    def __init__(self, input_dim, output_dim):\n",
    "        super(SoftmaxRegression, self).__init__()\n",
    "        self.fc = nn.Linear(input_dim, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)  # 展平输入\n",
    "        x = self.fc(x)\n",
    "        return torch.softmax(x, dim=1)\n",
    "\n",
    "# 定义超参数\n",
    "batch_size = 64\n",
    "epochs = 100  # 增加迭代周期的数量\n",
    "learning_rate = 0.01\n",
    "patience = 5  # 早停法的耐心值\n",
    "\n",
    "# 定义设备\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# 数据预处理\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,), (0.5,))\n",
    "])\n",
    "\n",
    "# 加载数据集（仅使用前 5000 个训练样本）\n",
    "train_dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transform)\n",
    "train_dataset = torch.utils.data.Subset(train_dataset, indices=range(5000))  # 快速测试\n",
    "test_dataset = datasets.MNIST(root='./data', train=False, download=True, transform=transform)\n",
    "\n",
    "# 划分训练集和验证集（80% 训练，20% 验证）\n",
    "train_size = int(0.8 * len(train_dataset))\n",
    "val_size = len(train_dataset) - train_size\n",
    "train_dataset, val_dataset = random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "# 创建 DataLoader\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "# 初始化模型、损失函数和优化器\n",
    "model = SoftmaxRegression(784, 10).to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "# 训练和验证\n",
    "best_val_accuracy = 0\n",
    "wait = 0\n",
    "for epoch in range(epochs):\n",
    "    model.train()\n",
    "    for data, target in train_loader:\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    # 验证集评估\n",
    "    model.eval()\n",
    "    val_correct = 0\n",
    "    val_total = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in val_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            _, predicted = torch.max(output, 1)\n",
    "            val_total += target.size(0)\n",
    "            val_correct += (predicted == target).sum().item()\n",
    "    val_accuracy = 100 * val_correct / val_total\n",
    "    print(f'Epoch {epoch + 1}, Val Accuracy: {val_accuracy:.2f}%')\n",
    "\n",
    "    # 早停法\n",
    "    if val_accuracy > best_val_accuracy:\n",
    "        best_val_accuracy = val_accuracy\n",
    "        wait = 0\n",
    "        best_model_wts = model.state_dict()\n",
    "    else:\n",
    "        wait += 1\n",
    "        if wait >= patience:\n",
    "            print(f'Early stopping at epoch {epoch + 1}')\n",
    "            break\n",
    "\n",
    "# 加载最佳模型权重\n",
    "model.load_state_dict(best_model_wts)\n",
    "\n",
    "# 测试集评估\n",
    "test_correct = 0\n",
    "test_total = 0\n",
    "with torch.no_grad():\n",
    "    for data, target in test_loader:\n",
    "        data, target = data.to(device), target.to(device)\n",
    "        output = model(data)\n",
    "        _, predicted = torch.max(output, 1)\n",
    "        test_total += target.size(0)\n",
    "        test_correct += (predicted == target).sum().item()\n",
    "test_accuracy = 100 * test_correct / test_total\n",
    "print(f'Test Accuracy: {test_accuracy:.2f}%')"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
